{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "70066de8-21a3-42eb-9aea-ef5d91abe761",
   "metadata": {},
   "source": [
    "#### 多线程版本 v3 三个并行方向\n",
    "1. 根目录子文件夹进行并行\n",
    "2. 通过os walk得到子文件夹的下属文件夹进行并行\n",
    "3. 对于文件夹下面的文件进行并行获取size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d4ccb344-305f-4b28-bec2-a683dfea0be0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import concurrent.futures\n",
    "import threading\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "98ccb046-3521-4124-aaf1-a6bd8873ab26",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def get_file_size0(file_path):\n",
    "    try:        \n",
    "        return os.path.getsize(file_path)\n",
    "    except FileNotFoundError:\n",
    "        return 0    \n",
    "\n",
    "def get_file_size(dirpath, files):\n",
    "    dir_size = 0\n",
    "    with ThreadPoolExecutor(max_workers=8) as executor:\n",
    "        futures = []\n",
    "        for file_name in files:\n",
    "            file_path = os.path.join(dirpath, file_name)\n",
    "            futures.append(executor.submit(get_file_size0, file_path))\n",
    "\n",
    "    for future in concurrent.futures.as_completed(futures):\n",
    "        dir_size += future.result()        \n",
    "    return dirpath, dir_size\n",
    "\n",
    "\n",
    "def get_dir_size(root,directory):\n",
    "    total_size = 0\n",
    "    base_stat={}\n",
    "    base_dir = os.path.join(root, directory)\n",
    "    \n",
    "    start_time = time.time()\n",
    "    with ThreadPoolExecutor(max_workers=8) as executor:\n",
    "        futures = []\n",
    "        for dirpath, _, files in os.walk(base_dir):\n",
    "             futures.append(executor.submit(get_file_size, dirpath, files))\n",
    "        # 获取结果\n",
    "        for future in concurrent.futures.as_completed(futures):\n",
    "            result = future.result()\n",
    "            dir_size = result[1]\n",
    "            total_size += dir_size\n",
    "            if dir_size > 200*1024**2:\n",
    "                base_stat[result[0]] = dir_size\n",
    "      \n",
    "    end_time = time.time() \n",
    "    print(f\"分析文件夹 {base_dir} 用时{(end_time - start_time):.2f} 秒 ---------------------\")\n",
    "    return directory, total_size, base_stat\n",
    "\n",
    "def scan_dirs(root):\n",
    "    print(f\"开始分析 Folder: {root} --------------------------------\")\n",
    "    directories = os.listdir(root) \n",
    "    statMap = {}\n",
    "    total_size = 0\n",
    "    with ThreadPoolExecutor(max_workers=8) as executor:\n",
    "        # 并发扫描多个目录\n",
    "        for size in executor.map(lambda x :get_dir_size(root,x), directories):\n",
    "            total_size += size[1]\n",
    "            statMap.update(size[2])\n",
    "        \n",
    "    #sorted_folder = dict(sorted(subfolder_sizes.items(), key=lambda item:item[1], reverse= True))\n",
    "         \n",
    "    return total_size,statMap\n",
    "\n",
    "def main(folder):\n",
    "    # 假设你有一个根目录或者一个包含多个子目录的列表\n",
    "    start_time = time.time()\n",
    "    result = scan_dirs(folder)\n",
    "    total_size = result[0]\n",
    "    statMap =  result[1]\n",
    "    end_time = time.time()\n",
    "    print(f\"{folder} Total size: {total_size / (1024 ** 3):.2f} GB, 分析耗时: {(end_time - start_time):.2f} 秒 \")\n",
    "    print(f\"开始打印瓶颈文件夹，总计 {len(statMap)}\")\n",
    "    # 打印瓶颈\n",
    "    sorted_folder = dict(sorted(statMap.items(), key=lambda item:item[1], reverse= True))    \n",
    "    for folder, size in sorted_folder.items():\n",
    "        print(f\"Folder: {folder}, size: {size / (1024**2):.2f} MB\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c89bf4b2-b495-41de-980d-5f7e2e812efe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始分析 Folder: C:\\Users --------------------------------\n",
      "分析文件夹 C:\\Users\\Default User 用时0.00 秒 ---------------------\n",
      "分析文件夹 C:\\Users\\desktop.ini 用时0.00 秒 ---------------------\n",
      "分析文件夹 C:\\Users\\Default 用时0.25 秒 ---------------------\n",
      "分析文件夹 C:\\Users\\Public 用时0.26 秒 ---------------------\n",
      "分析文件夹 C:\\Users\\All Users 用时4.57 秒 ---------------------\n",
      "分析文件夹 C:\\Users\\luischen 用时6.46 秒 ---------------------\n"
     ]
    }
   ],
   "source": [
    "folder = \"C:\\\\Users\"\n",
    "main(folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c1c4469-fdf3-42bc-8a0e-da96624a273c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61a48a81-f253-4ca9-9166-adb774217ad5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
